Load data

Got tables from appkernel.ccr.buffalo.edu modw database.

# cleanup
dbDisconnect(con)

Error in dbDisconnect(con) : object 'con' not found

# renaming things to make it more straightforward...?
all_res <- all_res1

Error: object 'all_res1' not found

# doesn't apply for me
# this section appears to determine what sort of node/cpu it ran on? So not exactly similar for me
edge_CPU_L5520 <-c(
sprintf("cpn-d14-%02d", seq(2,23)),
sprintf("cpn-d14-%02d", seq(26,39)),
sprintf("cpn-d15-%02d", c(seq(1,32),seq(34,40))),
sprintf("cpn-d16-%02d", c(seq(2,14),seq(17,41))),
"cpn-d15-33",
"cpn-d16-15",
"cpn-d16-16"
)

nodes_CPU_L5520 <- rep(FALSE,times=nrow(all_res))
for(i in 1:nrow(all_res)){
    #i<-1
    m_nodes<-strsplit(trimws(gsub(";"," ",all_res$nodes[[i]]))," ")[[1]]
    rigth_cpu <- TRUE
    for(m_node in m_nodes){
        #print(paste(m_node,m_node %in% edge_CPU_L5520))
        if(!(m_node %in% edge_CPU_L5520)){
            rigth_cpu=FALSE
        }
    }
    #print(paste(i,rigth_cpu))
    nodes_CPU_L5520[i]<-rigth_cpu
}
all_res$resource[nodes_CPU_L5520] <- "edge8core_ibm"
all_res$resource[all_res$resource=="edge"] <- "edge8core_dell"

all <- all_res %>% filter(app!="test") %>%
    select(state,task_id,resource,app,nnodes,walltime,collected,body)

# replacing on a conditional
all_res$cpu_type[grepl("cpn-u25-39",all_res$nodes)] <- "CPU-GOLD-6130"

Warning message:
In UseMethod("as.list") :
  call dbDisconnect() when finished working with a connection

# parses the body for all of the various metrics/info
metrics <- all %>% group_by(state,resource,app,nnodes,task_id) %>% do(get_metrics(.$body) )

Warning in .Internal(unique(x, incomparables, fromLast, nmax)) :
  call dbDisconnect() when finished working with a connection

|======================                                                                                                                                              | 14% ~13 s remaining    
|======================                                                                                                                                              | 14% ~13 s remaining    
|=======================                                                                                                                                             | 14% ~13 s remaining    
|========================                                                                                                                                            | 15% ~13 s remaining    
|========================                                                                                                                                            | 15% ~13 s remaining    
|=========================                                                                                                                                           | 16% ~13 s remaining    
|==========================                                                                                                                                          | 16% ~12 s remaining    
|===========================                                                                                                                                         | 16% ~12 s remaining    
|===========================                                                                                                                                         | 17% ~13 s remaining    
|============================                                                                                                                                        | 17% ~13 s remaining    
|============================                                                                                                                                        | 17% ~13 s remaining    
|=============================                                                                                                                                       | 18% ~13 s remaining    
|=============================                                                                                                                                       | 18% ~13 s remaining    
|==============================                                                                                                                                      | 18% ~13 s remaining    
|==============================                                                                                                                                      | 19% ~13 s remaining    
|===============================                                                                                                                                     | 19% ~13 s remaining    
|===============================                                                                                                                                     | 19% ~13 s remaining    
|================================                                                                                                                                    | 20% ~13 s remaining    
|================================                                                                                                                                    | 20% ~13 s remaining    
|=================================                                                                                                                                   | 20% ~13 s remaining    
|=================================                                                                                                                                   | 21% ~14 s remaining    
|==================================                                                                                                                                  | 21% ~14 s remaining    
|==================================                                                                                                                                  | 21% ~14 s remaining    
|===================================                                                                                                                                 | 22% ~14 s remaining    
|===================================                                                                                                                                 | 22% ~14 s remaining    
|====================================                                                                                                                                | 22% ~14 s remaining    
|====================================                                                                                                                                | 23% ~14 s remaining    
|=====================================                                                                                                                               | 23% ~14 s remaining    
|======================================                                                                                                                              | 23% ~14 s remaining    
|======================================                                                                                                                              | 24% ~14 s remaining    
|=======================================                                                                                                                             | 24% ~14 s remaining    
|=======================================                                                                                                                             | 24% ~14 s remaining    
|========================================                                                                                                                            | 24% ~14 s remaining    
|========================================                                                                                                                            | 25% ~14 s remaining    
|=========================================                                                                                                                           | 25% ~14 s remaining    
|=========================================                                                                                                                           | 25% ~14 s remaining    
|==========================================                                                                                                                          | 26% ~14 s remaining    
|==========================================                                                                                                                          | 26% ~14 s remaining    
|===========================================                                                                                                                         | 26% ~14 s remaining    
|===========================================                                                                                                                         | 27% ~14 s remaining    
|============================================                                                                                                                        | 27% ~14 s remaining    
|============================================                                                                                                                        | 27% ~14 s remaining    
|=============================================                                                                                                                       | 28% ~14 s remaining    
|=============================================                                                                                                                       | 28% ~14 s remaining    
|==============================================                                                                                                                      | 28% ~14 s remaining    
|==============================================                                                                                                                      | 28% ~14 s remaining    
|===============================================                                                                                                                     | 29% ~14 s remaining    
|===============================================                                                                                                                     | 29% ~14 s remaining    
|================================================                                                                                                                    | 29% ~14 s remaining    
|================================================                                                                                                                    | 30% ~14 s remaining    
|=================================================                                                                                                                   | 30% ~14 s remaining    
|=================================================                                                                                                                   | 30% ~14 s remaining    
|==================================================                                                                                                                  | 31% ~14 s remaining    
|==================================================                                                                                                                  | 31% ~14 s remaining    
|===================================================                                                                                                                 | 31% ~14 s remaining    
|===================================================                                                                                                                 | 32% ~14 s remaining    
|====================================================                                                                                                                | 32% ~14 s remaining    
|=====================================================                                                                                                               | 32% ~14 s remaining    
|=====================================================                                                                                                               | 33% ~14 s remaining    
|======================================================                                                                                                              | 33% ~14 s remaining    
|======================================================                                                                                                              | 33% ~14 s remaining    
|=======================================================                                                                                                             | 34% ~14 s remaining    
|=======================================================                                                                                                             | 34% ~13 s remaining    
|========================================================                                                                                                            | 34% ~13 s remaining    
|========================================================                                                                                                            | 35% ~13 s remaining    
|=========================================================                                                                                                           | 35% ~13 s remaining    
|=========================================================                                                                                                           | 35% ~13 s remaining    
|==========================================================                                                                                                          | 36% ~13 s remaining    
|==========================================================                                                                                                          | 36% ~13 s remaining    
|===========================================================                                                                                                         | 36% ~13 s remaining    
|===========================================================                                                                                                         | 36% ~13 s remaining    
|============================================================                                                                                                        | 37% ~13 s remaining    
|============================================================                                                                                                        | 37% ~13 s remaining    
|=============================================================                                                                                                       | 37% ~13 s remaining    
|=============================================================                                                                                                       | 38% ~13 s remaining    
|==============================================================                                                                                                      | 38% ~13 s remaining    
|==============================================================                                                                                                      | 38% ~13 s remaining    
|===============================================================                                                                                                     | 39% ~13 s remaining    
|===============================================================                                                                                                     | 39% ~13 s remaining    
|================================================================                                                                                                    | 39% ~13 s remaining    
|================================================================                                                                                                    | 40% ~13 s remaining    
|=================================================================                                                                                                   | 40% ~13 s remaining    
|=================================================================                                                                                                   | 40% ~13 s remaining    
|==================================================================                                                                                                  | 40% ~13 s remaining    
|==================================================================                                                                                                  | 41% ~13 s remaining    
|===================================================================                                                                                                 | 41% ~13 s remaining    
|===================================================================                                                                                                 | 41% ~13 s remaining    
|====================================================================                                                                                                | 42% ~12 s remaining    
|=====================================================================                                                                                               | 42% ~12 s remaining    
|=====================================================================                                                                                               | 42% ~12 s remaining    
|======================================================================                                                                                              | 43% ~12 s remaining    
|======================================================================                                                                                              | 43% ~12 s remaining    
|=======================================================================                                                                                             | 43% ~12 s remaining    
|=======================================================================                                                                                             | 44% ~12 s remaining    
|========================================================================                                                                                            | 44% ~12 s remaining    
|========================================================================                                                                                            | 44% ~12 s remaining    
|=========================================================================                                                                                           | 45% ~12 s remaining    
|=========================================================================                                                                                           | 45% ~12 s remaining    
|==========================================================================                                                                                          | 45% ~12 s remaining    
|==========================================================================                                                                                          | 46% ~12 s remaining    
|===========================================================================                                                                                         | 46% ~12 s remaining    
|===========================================================================                                                                                         | 46% ~12 s remaining    
|============================================================================                                                                                        | 47% ~12 s remaining    
|============================================================================                                                                                        | 47% ~12 s remaining    
|=============================================================================                                                                                       | 47% ~11 s remaining    
|=============================================================================                                                                                       | 48% ~11 s remaining    
|==============================================================================                                                                                      | 48% ~11 s remaining    
|==============================================================================                                                                                      | 48% ~11 s remaining    
|===============================================================================                                                                                     | 48% ~11 s remaining    
|================================================================================                                                                                    | 49% ~11 s remaining    
|================================================================================                                                                                    | 49% ~11 s remaining    
|=================================================================================                                                                                   | 49% ~11 s remaining    
|=================================================================================                                                                                   | 50% ~11 s remaining    
|==================================================================================                                                                                  | 50% ~11 s remaining    
|==================================================================================                                                                                  | 50% ~11 s remaining    
|===================================================================================                                                                                 | 51% ~11 s remaining    
|====================================================================================                                                                                | 51% ~11 s remaining    
|=====================================================================================                                                                               | 52% ~10 s remaining    
|=====================================================================================                                                                               | 52% ~10 s remaining    
|======================================================================================                                                                              | 53% ~10 s remaining    
|=======================================================================================                                                                             | 53% ~10 s remaining    
|=======================================================================================                                                                             | 54% ~10 s remaining    
|========================================================================================                                                                            | 54% ~10 s remaining    
|=========================================================================================                                                                           | 54% ~10 s remaining    
|==========================================================================================                                                                          | 55% ~10 s remaining    
|==========================================================================================                                                                          | 55% ~9 s remaining     
|===========================================================================================                                                                         | 56% ~9 s remaining     
|============================================================================================                                                                        | 56% ~9 s remaining     
|============================================================================================                                                                        | 57% ~9 s remaining     
|=============================================================================================                                                                       | 57% ~9 s remaining     
|==============================================================================================                                                                      | 57% ~9 s remaining     
|==============================================================================================                                                                      | 58% ~9 s remaining     
|===============================================================================================                                                                     | 58% ~9 s remaining     
|================================================================================================                                                                    | 59% ~9 s remaining     
|=================================================================================================                                                                   | 59% ~8 s remaining     
|==================================================================================================                                                                  | 60% ~8 s remaining     
|==================================================================================================                                                                  | 60% ~8 s remaining     
|===================================================================================================                                                                 | 61% ~8 s remaining     
|====================================================================================================                                                                | 61% ~8 s remaining     
|====================================================================================================                                                                | 61% ~8 s remaining     
|=====================================================================================================                                                               | 62% ~8 s remaining     
|======================================================================================================                                                              | 62% ~8 s remaining     
|=======================================================================================================                                                             | 63% ~8 s remaining     
|=======================================================================================================                                                             | 63% ~7 s remaining     
|========================================================================================================                                                            | 64% ~7 s remaining     
|=========================================================================================================                                                           | 64% ~7 s remaining     
|==========================================================================================================                                                          | 65% ~7 s remaining     
|===========================================================================================================                                                         | 65% ~7 s remaining     
|===========================================================================================================                                                         | 66% ~7 s remaining     
|============================================================================================================                                                        | 66% ~7 s remaining     
|=============================================================================================================                                                       | 67% ~7 s remaining     
|=============================================================================================================                                                       | 67% ~7 s remaining     
|==============================================================================================================                                                      | 67% ~7 s remaining     
|==============================================================================================================                                                      | 68% ~7 s remaining     
|===============================================================================================================                                                     | 68% ~7 s remaining     
|===============================================================================================================                                                     | 68% ~6 s remaining     
|================================================================================================================                                                    | 68% ~6 s remaining     
|================================================================================================================                                                    | 69% ~6 s remaining     
|=================================================================================================================                                                   | 69% ~6 s remaining     
|=================================================================================================================                                                   | 69% ~6 s remaining     
|==================================================================================================================                                                  | 70% ~6 s remaining     
|==================================================================================================================                                                  | 70% ~6 s remaining     
|===================================================================================================================                                                 | 70% ~6 s remaining     
|===================================================================================================================                                                 | 71% ~6 s remaining     
|====================================================================================================================                                                | 71% ~6 s remaining     
|=====================================================================================================================                                               | 71% ~6 s remaining     
|=====================================================================================================================                                               | 72% ~6 s remaining     
|======================================================================================================================                                              | 72% ~6 s remaining     
|======================================================================================================================                                              | 72% ~6 s remaining     
|=======================================================================================================================                                             | 73% ~6 s remaining     
|=======================================================================================================================                                             | 73% ~6 s remaining     
|========================================================================================================================                                            | 73% ~6 s remaining     
|========================================================================================================================                                            | 74% ~5 s remaining     
|=========================================================================================================================                                           | 74% ~5 s remaining     
|=========================================================================================================================                                           | 74% ~5 s remaining     
|==========================================================================================================================                                          | 74% ~5 s remaining     
|==========================================================================================================================                                          | 75% ~5 s remaining     
|===========================================================================================================================                                         | 75% ~5 s remaining     
|===========================================================================================================================                                         | 75% ~5 s remaining     
|============================================================================================================================                                        | 76% ~5 s remaining     
|============================================================================================================================                                        | 76% ~5 s remaining     
|=============================================================================================================================                                       | 76% ~5 s remaining     
|=============================================================================================================================                                       | 77% ~5 s remaining     
|==============================================================================================================================                                      | 77% ~5 s remaining     
|==============================================================================================================================                                      | 77% ~5 s remaining     
|===============================================================================================================================                                     | 78% ~5 s remaining     
|===============================================================================================================================                                     | 78% ~5 s remaining     
|================================================================================================================================                                    | 78% ~5 s remaining     
|================================================================================================================================                                    | 78% ~5 s remaining     
|=================================================================================================================================                                   | 79% ~4 s remaining     
|=================================================================================================================================                                   | 79% ~4 s remaining     
|==================================================================================================================================                                  | 79% ~4 s remaining     
|==================================================================================================================================                                  | 80% ~4 s remaining     
|===================================================================================================================================                                 | 80% ~4 s remaining     
|===================================================================================================================================                                 | 80% ~4 s remaining     
|====================================================================================================================================                                | 81% ~4 s remaining     
|====================================================================================================================================                                | 81% ~4 s remaining     
|=====================================================================================================================================                               | 81% ~4 s remaining     
|=====================================================================================================================================                               | 82% ~4 s remaining     
|======================================================================================================================================                              | 82% ~4 s remaining     
|======================================================================================================================================                              | 82% ~4 s remaining     
|=======================================================================================================================================                             | 83% ~4 s remaining     
|=======================================================================================================================================                             | 83% ~4 s remaining     
|========================================================================================================================================                            | 83% ~4 s remaining     
|========================================================================================================================================                            | 83% ~4 s remaining     
|=========================================================================================================================================                           | 84% ~3 s remaining     
|=========================================================================================================================================                           | 84% ~3 s remaining     
|==========================================================================================================================================                          | 84% ~3 s remaining     
|==========================================================================================================================================                          | 85% ~3 s remaining     
|===========================================================================================================================================                         | 85% ~3 s remaining     
|===========================================================================================================================================                         | 85% ~3 s remaining     
|============================================================================================================================================                        | 86% ~3 s remaining     
|============================================================================================================================================                        | 86% ~3 s remaining     
|=============================================================================================================================================                       | 86% ~3 s remaining     
|=============================================================================================================================================                       | 87% ~3 s remaining     
|==============================================================================================================================================                      | 87% ~3 s remaining     
|===============================================================================================================================================                     | 87% ~3 s remaining     
|===============================================================================================================================================                     | 88% ~3 s remaining     
|================================================================================================================================================                    | 88% ~3 s remaining     
|================================================================================================================================================                    | 88% ~3 s remaining     
|=================================================================================================================================================                   | 88% ~2 s remaining     
|=================================================================================================================================================                   | 89% ~2 s remaining     
|=================================================================================================================================================                   | 89% ~2 s remaining     
|==================================================================================================================================================                  | 89% ~2 s remaining     
|==================================================================================================================================================                  | 90% ~2 s remaining     
|===================================================================================================================================================                 | 90% ~2 s remaining     
|===================================================================================================================================================                 | 90% ~2 s remaining     
|====================================================================================================================================================                | 91% ~2 s remaining     
|=====================================================================================================================================================               | 91% ~2 s remaining     
|=====================================================================================================================================================               | 91% ~2 s remaining     
|=====================================================================================================================================================               | 91% ~2 s remaining     
|======================================================================================================================================================              | 92% ~2 s remaining     
|======================================================================================================================================================              | 92% ~2 s remaining     
|=======================================================================================================================================================             | 92% ~2 s remaining     
|=======================================================================================================================================================             | 93% ~2 s remaining     
|========================================================================================================================================================            | 93% ~2 s remaining     
|========================================================================================================================================================            | 93% ~1 s remaining     
|=========================================================================================================================================================           | 94% ~1 s remaining     
|==========================================================================================================================================================          | 94% ~1 s remaining     
|==========================================================================================================================================================          | 94% ~1 s remaining     
|===========================================================================================================================================================         | 95% ~1 s remaining     
|===========================================================================================================================================================         | 95% ~1 s remaining     
|============================================================================================================================================================        | 95% ~1 s remaining     
|============================================================================================================================================================        | 96% ~1 s remaining     
|=============================================================================================================================================================       | 96% ~1 s remaining     
|=============================================================================================================================================================       | 96% ~1 s remaining     
|==============================================================================================================================================================      | 96% ~1 s remaining     
|==============================================================================================================================================================      | 97% ~1 s remaining     
|===============================================================================================================================================================     | 97% ~1 s remaining     
|===============================================================================================================================================================     | 97% ~1 s remaining     
|================================================================================================================================================================    | 98% ~1 s remaining     
|================================================================================================================================================================    | 98% ~0 s remaining     
|=================================================================================================================================================================   | 98% ~0 s remaining     
|=================================================================================================================================================================   | 98% ~0 s remaining     
|=================================================================================================================================================================   | 99% ~0 s remaining     
|==================================================================================================================================================================  | 99% ~0 s remaining     
|==================================================================================================================================================================  | 99% ~0 s remaining     
|=================================================================================================================================================================== |100% ~0 s remaining     
|====================================================================================================================================================================|100% ~0 s remaining     

# gets only the entries that are hpcc
all_hpcc <- all[all$app == "HPCC",]
all_hpcc$task_id <- as.numeric(all_hpcc$task_id)
metrics_hpcc <- metrics[metrics$app == "HPCC",]

# keeps the units of metrics intact
metrics_hpcc$metric_and_units <- paste(metrics_hpcc$metric, "(", metrics_hpcc$units,")", sep="")
remove(temp)

# makes a data frame with 21 columns, 25 rows (however many metrics there are)
temp <- data.frame(matrix(ncol=length(unique(metrics_hpcc$metric_and_units)), nrow=length(unique(metrics_hpcc$task_id))))

# sets up the name of the columns (based on the metrics gotten)
column_names = c("task_id", unique(metrics_hpcc$metric_and_units))
colnames(temp) <- column_names

Error in names(x) <- value : 
  'names' attribute [21] must be the same length as the vector [20]

full_gamess <- setup_data("GAMESS")
full_gamess <- setup_data("GAMESS")
full_hpcg <- setup_data("HPCG") 
full_hpcc <- setup_data("HPCC")
full_hpcg <- setup_data("HPCG") 
full_hpcc <- setup_data("HPCC")
full_ior <- setup_data("IOR")

Error in `$<-.data.frame`(`*tmp*`, "metric_and_units", value = "()") : 
  replacement has 1 row, data has 0

t.test(hpcc_singularity$`Fast Fourier Transform (FFTW) Floating-Point Performance(MFLOP per Second)`, hpcc_bare_metal$`Fast Fourier Transform (FFTW) Floating-Point Performance(MFLOP per Second)`)

    Welch Two Sample t-test

data:  hpcc_singularity$`Fast Fourier Transform (FFTW) Floating-Point Performance(MFLOP per Second)` and hpcc_bare_metal$`Fast Fourier Transform (FFTW) Floating-Point Performance(MFLOP per Second)`
t = -3.6828, df = 207.04, p-value = 0.0002944
alternative hypothesis: true difference in means is not equal to 0
95 percent confidence interval:
 -510.4233 -154.4815
sample estimates:
mean of x mean of y 
 19772.84  20105.30 

ggplot(full_hpcc, aes(x = resource, y = walltime, fill=cpu_type)) +
    geom_dotplot(binaxis='y', stackdir='centerwhole')

colnames(full_hpcc) <- gsub(" ","_",colnames(full_hpcc))
ggplot(full_hpcc, aes(x = resource, y = `MPI Random Access(MUpdate per Second)`, fill=cpu_type)) +
    geom_dotplot(binaxis='y', stackdir='center', binwidth=0.1)

ggplot(full_hpcc, aes(x = resource, y = `Average Double-Precision General Matrix Multiplication (DGEMM) Floating-Point Performance(MFLOP per Second)`, fill=cpu_type)) +
    geom_dotplot(binaxis='y', stackdir='center')

ggplot(full_hpcc, aes(x = resource, y = `Average STREAM 'Add' Memory Bandwidth(MByte per Second)`, fill=cpu_type)) +
    geom_dotplot(binaxis='y', stackdir='center')

ggplot(full_hpcc, aes(x = resource, y = `Average STREAM 'Copy' Memory Bandwidth(MByte per Second)`, fill=cpu_type)) +
    geom_dotplot(binaxis='y', stackdir='center')

ggplot(full_hpcc, aes(x = resource, y = `Fast Fourier Transform (FFTW) Floating-Point Performance(MFLOP per Second)`, fill=cpu_type)) +
    geom_dotplot(binaxis='y', stackdir='center')

ggplot(full_hpcc, aes(x = resource, y = `High Performance LINPACK Efficiency(Percent)`, fill=cpu_type)) +
    geom_dotplot(binaxis='y', stackdir='center')

ggplot(full_hpcc, aes(x = resource, y = `Parallel Matrix Transpose (PTRANS)(MByte per Second)`, fill=cpu_type)) +
    geom_dotplot(binaxis='y', stackdir='center')

# making sure to filter out any resources we aint using
metrics %<>% filter(resource %in% c("vortex_bare_metal","vortex_dock_sing"))

metrics$resource <- factor(
    metrics$resource,
    levels = unique(c("vortex_bare_metal","vortex_dock_sing")))

levels(metrics$resource) <- c("TestCluster","ProductionCluster")
metrics$nnodes <- factor(metrics$nnodes,levels = unique(c(1,2,4,8,16,metrics$nnodes)))

metrics$state_id <- metrics$state
metrics$state <- as.character(metrics$state_id)
metrics$state[metrics$state=="1"]<-"Before"
metrics$state[metrics$state=="2"]<-"After"

metrics$state <- factor(metrics$state,levels = c("Before","After"))

metrics %<>% arrange(collected)

#convert md performance should be ns/day was sim second per Day
j <- metrics$metric=="Molecular Dynamics Performance"
metrics$value[j] <- metrics$value[j]*1e9
metrics$units[j] <- "ns/day"

write.csv(metrics,"metrics.csv")
save(
    list=c("metrics"),
    file="metrics.Rdata"
)

metrics %<>% filter(app %in% c("HPCC","NAMD"))
metrics$app <- factor(as.character(metrics$app),levels = c("HPCC","NAMD"))

selected_metrics <- c(
    "Wall Clock Time",
    "Molecular Dynamics Simulation Performance",
    "High Performance LINPACK Floating-Point Performance",
    "Fast Fourier Transform (FFTW) Floating-Point Performance",
    "Average Double-Precision General Matrix Multiplication (DGEMM) Floating-Point Performance",
    "Average STREAM 'Copy' Memory Bandwidth"
)
metrics %<>% filter(metric %in% selected_metrics)
metrics$metric <- factor(as.character(metrics$metric),levels = selected_metrics)
metrics$metric_long <- metrics$metric
levels(metrics$metric) <- c(
    "Wall Time",
    "MD Perf.",
    "LINPACK",
    "FFT",
    "DGEMM",
    "STREAM"
)

#convert md performance should be ns/day was sim second per Day
j <- metrics$metric=="Molecular Dynamics Performance"
metrics$value[j] <- metrics$value[j]*1e9

j <- metrics$metric=="LINPACK"
metrics$value[j] <- metrics$value[j]/1024

#metrics %>% group_by(metric) %>% summarise(units=last(units))
metrics$units <- metrics$metric
levels(metrics$units) <- c(
    "s",
    "ns/day",
    "GFLOP/s",
    "MFLOP/s",
    "MFLOP/s",
    "MB/s"
)

save(
    list=c("metrics"),
    file="metrics_selected.Rdata"
)

load(file="metrics_selected.Rdata")
#select only what is needed2

#only up to 30 points for Testing-Cluster
metrics %>% group_by(resource, app, nnodes, metric, state) %>% 
    mutate(id = row_number()) %>% 
    summarise(n=n(),avg=mean(value),units=last(units))
    
metrics %<>% group_by(resource, app, nnodes, metric, state) %>% 
        mutate(id = row_number()) %>% ungroup()


metrics %<>% filter(!(resource=="Testing-Cluster" & id > 30L))

#Sparcify After for UBHPC
metrics %>% filter(resource=="UB-HPC" & app == "NAMD") %>% group_by(resource, app, nnodes, metric, state) %>% 
    mutate(id = row_number()) %>% 
    summarise(n=n(),avg=mean(value),units=last(units))

metrics_ubhpc <- metrics %>% filter(resource=="UB-HPC" & state=="After" & metric == "Wall Time" & app == "NAMD" & nnodes == 1L & id > 10)
set.seed(08122018)
remover_task_id <- metrics_ubhpc$task_id[!(metrics_ubhpc$task_id %in% sample(metrics_ubhpc$task_id,size = 20L))]
metrics %<>% filter(!(task_id %in% remover_task_id))

metrics_ubhpc <- metrics %>% filter(resource=="UB-HPC" & state=="After" & metric == "Wall Time" & app == "NAMD" & nnodes == 2L & id > 20)
remover_task_id <- metrics_ubhpc$task_id[!(metrics_ubhpc$task_id %in% sample(metrics_ubhpc$task_id,size = 10L))]
metrics %<>% filter(!(task_id %in% remover_task_id))


metrics_ubhpc <- metrics %>% filter(resource=="UB-HPC" & state=="After" & metric == "Wall Time" & app == "HPCC" & nnodes == 1L & id > 10)
set.seed(08122018)
remover_task_id <- metrics_ubhpc$task_id[!(metrics_ubhpc$task_id %in% sample(metrics_ubhpc$task_id,size = 20L))]
metrics %<>% filter(!(task_id %in% remover_task_id))

metrics_ubhpc <- metrics %>% filter(resource=="UB-HPC" & state=="After" & metric == "Wall Time" & app == "HPCC" & nnodes == 2L & id > 10)
remover_task_id <- metrics_ubhpc$task_id[!(metrics_ubhpc$task_id %in% sample(metrics_ubhpc$task_id,size = 20L))]
metrics %<>% filter(!(task_id %in% remover_task_id))


metrics %>% filter(resource=="UB-HPC") %>% group_by(resource, app, nnodes, metric, state) %>% 
    mutate(id = row_number()) %>% 
    summarise(n=n(),avg=mean(value),units=last(units))

save(
    list=c("metrics"),
    file="metrics_selected2.Rdata"
)

#Read metrics bigger smaller

ak_metrics <- read.csv("ak_metrics.csv",header = F,col.names = c("app","metric","better"))

m_app<-ak_metrics$app[[1]]
for(i in 2:nrow(ak_metrics)){
    if(ak_metrics$app[[i]]==""){
        ak_metrics$app[[i]]=m_app
    }else{
        m_app<-ak_metrics$app[[i]]
    }
}
ak_metrics$BiggerBetter <- ak_metrics$better !="S"

ak_metrics %<>% filter(metric!="" & app %in% unlist(app_conv,use.names = F))  %>% select(-better)

save(
    list=c("ak_metrics"),
    file="ak_metrics.Rdata"
)